<!DOCTYPE html>





<html lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 3.9.0">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/favicon.ico?v=7.4.0">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon.ico?v=7.4.0">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon.ico?v=7.4.0">
  <link rel="mask-icon" href="/images/logo.svg?v=7.4.0" color="#222">

<link rel="stylesheet" href="/css/main.css?v=7.4.0">


<link rel="stylesheet" href="/lib/font-awesome/css/font-awesome.min.css?v=4.7.0">


<script id="hexo-configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Gemini',
    version: '7.4.0',
    exturl: false,
    sidebar: {"position":"left","display":"post","offset":12,"onmobile":false,"dimmer":false},
    copycode: {"enable":false,"show_result":false,"style":null},
    back2top: {"enable":true,"sidebar":false,"scrollpercent":false},
    bookmark: {"enable":false,"color":"#222","save":"auto"},
    fancybox: false,
    mediumzoom: false,
    lazyload: false,
    pangu: false,
    algolia: {
      appID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    },
    localsearch: {"enable":true,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},
    path: 'search.json',
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    translation: {
      copy_button: '复制',
      copy_success: '复制成功',
      copy_failure: '复制失败'
    },
    sidebarPadding: 40
  };
</script>

  <meta name="description" content="0. 前言Chrome59(linux、macos)、 Chrome60(windows)之后，Chrome自带headless(无界面)模式很方便做自动化测试或者爬虫。但是如何和headless模式的Chrome交互则是一个问题。通过启动Chrome时的命令行参数仅能实现简易的启动时初始化操作。Selenium、Webdriver等是一种解决方案，但是往往依赖众多，不够扁平。 puppeteer">
<meta name="keywords" content="python,pyppeteer,爬虫">
<meta property="og:type" content="article">
<meta property="og:title" content="python爬虫利器pyppeteer的使用">
<meta property="og:url" content="https://unix2dos.github.io/p/6b1ba1b4.html">
<meta property="og:site_name" content="Levon&#39;s Blog">
<meta property="og:description" content="0. 前言Chrome59(linux、macos)、 Chrome60(windows)之后，Chrome自带headless(无界面)模式很方便做自动化测试或者爬虫。但是如何和headless模式的Chrome交互则是一个问题。通过启动Chrome时的命令行参数仅能实现简易的启动时初始化操作。Selenium、Webdriver等是一种解决方案，但是往往依赖众多，不够扁平。 puppeteer">
<meta property="og:locale" content="zh-CN">
<meta property="og:updated_time" content="2019-09-27T15:36:51.113Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="python爬虫利器pyppeteer的使用">
<meta name="twitter:description" content="0. 前言Chrome59(linux、macos)、 Chrome60(windows)之后，Chrome自带headless(无界面)模式很方便做自动化测试或者爬虫。但是如何和headless模式的Chrome交互则是一个问题。通过启动Chrome时的命令行参数仅能实现简易的启动时初始化操作。Selenium、Webdriver等是一种解决方案，但是往往依赖众多，不够扁平。 puppeteer">
  <link rel="canonical" href="https://unix2dos.github.io/p/6b1ba1b4">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome: false,
    isPost: true,
    isPage: false,
    isArchive: false
  };
</script>

  <title>python爬虫利器pyppeteer的使用 | Levon's Blog</title>
  








  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .logo,
  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-CN">
  <div class="container use-motion">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-meta">

    <div>
      <a href="/" class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">Levon's Blog</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
        <h1 class="site-subtitle" itemprop="description">微信:  L6241425</h1>
      
  </div>

  <div class="site-nav-toggle">
    <button aria-label="切换导航栏">
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>


<nav class="site-nav">
  
  <ul id="menu" class="menu">
      
      
      
        
        <li class="menu-item menu-item-home">
      
    

    <a href="/" rel="section"><i class="menu-item-icon fa fa-fw fa-home"></i> <br>首页</a>

  </li>
      
      
      
        
        <li class="menu-item menu-item-archives">
      
    

    <a href="/archives/" rel="section"><i class="menu-item-icon fa fa-fw fa-archive"></i> <br>时光</a>

  </li>
      
      
      
        
        <li class="menu-item menu-item-categories">
      
    

    <a href="/categories/" rel="section"><i class="menu-item-icon fa fa-fw fa-th"></i> <br>分类</a>

  </li>
      
      
      
        
        <li class="menu-item menu-item-tags">
      
    

    <a href="/tags/" rel="section"><i class="menu-item-icon fa fa-fw fa-tags"></i> <br>标签</a>

  </li>
      <li class="menu-item menu-item-search">
        <a href="javascript:;" class="popup-trigger">
        
          <i class="menu-item-icon fa fa-search fa-fw"></i> <br>搜索</a>
      </li>
    
  </ul>

</nav>
  <div class="site-search">
    <div class="popup search-popup">
    <div class="search-header">
  <span class="search-icon">
    <i class="fa fa-search"></i>
  </span>
  <div class="search-input-container">
    <input autocomplete="off" autocorrect="off" autocapitalize="none"
           placeholder="搜索..." spellcheck="false"
           type="text" id="search-input">
  </div>
  <span class="popup-btn-close">
    <i class="fa fa-times-circle"></i>
  </span>
</div>
<div id="search-result"></div>

</div>
<div class="search-pop-overlay"></div>

  </div>
</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>


    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
            

          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
      <article itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block post">
    <link itemprop="mainEntityOfPage" href="https://unix2dos.github.io/p/6b1ba1b4.html">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="Levon">
      <meta itemprop="description" content="Never Give Up">
      <meta itemprop="image" content="/images/avatar.jpg">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Levon's Blog">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">python爬虫利器pyppeteer的使用

          
        </h2>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
                
              

              <time title="创建时间：2019-08-31 12:00:46" itemprop="dateCreated datePublished" datetime="2019-08-31T12:00:46+08:00">2019-08-31</time>
            </span>
          
            

            
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2019-09-27 23:36:51" itemprop="dateModified" datetime="2019-09-27T23:36:51+08:00">2019-09-27</time>
              </span>
            
          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing"><a href="/categories/1-编程语言/" itemprop="url" rel="index"><span itemprop="name">1-编程语言</span></a></span>

                
                
                  ，
                
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing"><a href="/categories/1-编程语言/python/" itemprop="url" rel="index"><span itemprop="name">python</span></a></span>

                
                
              
            </span>
          

          
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="fa fa-comment-o"></i>
      </span>
        
      
      <span class="post-meta-item-text">Disqus：</span>
    
    <a title="disqus" href="/p/6b1ba1b4.html#comments" itemprop="discussionUrl"><span class="post-comments-count disqus-comment-count" data-disqus-identifier="p/6b1ba1b4.html" itemprop="commentCount"></span></a>
  </span>
  
  
          <br>
            <span class="post-meta-item" title="本文字数">
              <span class="post-meta-item-icon">
                <i class="fa fa-file-word-o"></i>
              </span>
              
                <span class="post-meta-item-text">本文字数：</span>
              
              <span>3.4k</span>
            </span>
          
            <span class="post-meta-item" title="阅读时长">
              <span class="post-meta-item-icon">
                <i class="fa fa-clock-o"></i>
              </span>
              
                <span class="post-meta-item-text">阅读时长 &asymp;</span>
              
              <span>3 分钟</span>
            </span>
          

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <h3 id="0-前言"><a href="#0-前言" class="headerlink" title="0. 前言"></a>0. 前言</h3><p>Chrome59(linux、macos)、 Chrome60(windows)之后，Chrome自带<a href="https://developers.google.com/web/updates/2017/04/headless-chrome" target="_blank" rel="noopener">headless(无界面)模式</a>很方便做自动化测试或者爬虫。但是如何和headless模式的Chrome交互则是一个问题。通过启动Chrome时的命令行参数仅能实现简易的启动时初始化操作。Selenium、Webdriver等是一种解决方案，但是往往依赖众多，不够扁平。</p>
<p>puppeteer是谷歌官方出品的一个通过DevTools协议控制headless Chrome的Node库。可以通过puppeteer的提供的api直接控制Chrome模拟大部分用户操作来进行UI Test或者作为爬虫访问页面来收集数据。</p>
<p>pyperteer是puppeteer的Python实现，相比于selenium具有异步加载、速度快、具备有界面/无界面模式、伪装性更强不易被识别为机器人同时可以伪装手机平板等终端；但是也有一些缺点，如接口不易理解、语义晦涩；</p>
<a id="more"></a>



<h3 id="1-pyppeteer使用"><a href="#1-pyppeteer使用" class="headerlink" title="1. pyppeteer使用"></a>1. pyppeteer使用</h3><figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pip3 install pyppeteer</span><br></pre></td></tr></table></figure>

<h5 id="1-1-无头模式"><a href="#1-1-无头模式" class="headerlink" title="1.1 无头模式"></a>1.1 无头模式</h5><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">browser = <span class="keyword">await</span> launch(&#123;<span class="string">'headless'</span>: <span class="literal">False</span>, <span class="string">'args'</span>: [<span class="string">'--no-sandbox'</span>]&#125;)</span><br></pre></td></tr></table></figure>

<p>headless=True, 不弹出浏览器, 测试阶段可以设置为 False 观测</p>
<h5 id="1-2-page方法"><a href="#1-2-page方法" class="headerlink" title="1.2 page方法"></a>1.2 page方法</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">page.click 点击</span><br><span class="line">page.type 输入</span><br><span class="line">page.select 下拉框</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">page.click(<span class="string">'.rfm input[name="cookietime"]'</span>)</span><br></pre></td></tr></table></figure>

<p>对于 page方法内的选择元素语法请参考:  <a href="https://www.w3schools.com/cssref/css_selectors.asp" target="_blank" rel="noopener">https://www.w3schools.com/cssref/css_selectors.asp</a></p>
<h3 id="2-pyppeteer-linux运行问题"><a href="#2-pyppeteer-linux运行问题" class="headerlink" title="2. pyppeteer  linux运行问题"></a>2. pyppeteer  linux运行问题</h3><h5 id="2-1-centos无法运行pyppeteer"><a href="#2-1-centos无法运行pyppeteer" class="headerlink" title="2.1 centos无法运行pyppeteer"></a>2.1 centos无法运行pyppeteer</h5><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">yum -y install libX11 libXcomposite libXcursor libXdamage libXext libXi libXtst cups-libs libXScrnSaver libXrandr alsa-lib pango atk at-spi2-atk gtk3</span><br></pre></td></tr></table></figure>

<h5 id="2-2-Bad-NaCl-helper-startup-ack"><a href="#2-2-Bad-NaCl-helper-startup-ack" class="headerlink" title="2.2 Bad NaCl helper startup ack"></a>2.2 Bad NaCl helper startup ack</h5><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">ERROR:nacl_fork_delegate_linux.cc(314)] Bad NaCl helper startup ack (0 bytes)\n\n(chrome:24935)</span><br></pre></td></tr></table></figure>

<p>使用无头模式</p>
<h5 id="2-3-Navigation-Timeout-Exceeded-30000-ms-exceeded"><a href="#2-3-Navigation-Timeout-Exceeded-30000-ms-exceeded" class="headerlink" title="2.3 Navigation Timeout Exceeded: 30000 ms exceeded"></a>2.3 Navigation Timeout Exceeded: 30000 ms exceeded</h5><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">await page.goto(&quot;https://www.baidu.com&quot;, timeout=0)</span><br></pre></td></tr></table></figure>

<ul>
<li><p>加上timeout</p>
</li>
<li><p>检测封禁 ip</p>
</li>
</ul>
<h3 id="3-pyppeteer-问题解决方案"><a href="#3-pyppeteer-问题解决方案" class="headerlink" title="3. pyppeteer 问题解决方案"></a>3. pyppeteer 问题解决方案</h3><h5 id="3-1-抓取js-渲染后的数据"><a href="#3-1-抓取js-渲染后的数据" class="headerlink" title="3.1 抓取js 渲染后的数据"></a>3.1 抓取js 渲染后的数据</h5><p>使用 requests 是无法正常抓取到相关数据的。因为什么？因为这个页面是 JavaScript 渲染而成的，我们所看到的内容都是网页加载后又执行了 JavaScript 之后才呈现出来的，因此这些条目数据并不存在于原始 HTML 代码中，而 requests 仅仅抓取的是原始 HTML 代码。</p>
<p>好的，所以遇到这种类型的网站我们应该怎么办呢？</p>
<p>其实答案有很多：</p>
<ul>
<li>分析网页源代码数据，如果数据是隐藏在 HTML 中的其他地方，以 JavaScript 变量的形式存在，直接提取就好了。</li>
<li>分析 Ajax，很多数据可能是经过 Ajax 请求时候获取的，所以可以分析其接口。</li>
<li>模拟 JavaScript 渲染过程，直接抓取渲染后的结果。</li>
</ul>
<p>而 Pyppeteer 和 Selenium 就是用的第三种方法，下面我们再用 Pyppeteer 来试试，如果用 Pyppeteer 实现如上页面的抓取的话，代码就可以写为如下形式：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">from</span> pyppeteer <span class="keyword">import</span> launch</span><br><span class="line"><span class="keyword">from</span> pyquery <span class="keyword">import</span> PyQuery <span class="keyword">as</span> pq</span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">def</span> <span class="title">main</span><span class="params">()</span>:</span></span><br><span class="line">    browser = <span class="keyword">await</span> launch()</span><br><span class="line">    page = <span class="keyword">await</span> browser.newPage()</span><br><span class="line">    <span class="keyword">await</span> page.goto(<span class="string">'http://quotes.toscrape.com/js/'</span>)</span><br><span class="line">    doc = pq(<span class="keyword">await</span> page.content())</span><br><span class="line">    print(<span class="string">'Quotes:'</span>, doc(<span class="string">'.quote'</span>).length)</span><br><span class="line">    <span class="keyword">await</span> browser.close()</span><br><span class="line"></span><br><span class="line">asyncio.get_event_loop().run_until_complete(main())</span><br></pre></td></tr></table></figure>

<h5 id="3-2-webdriver-检测问题"><a href="#3-2-webdriver-检测问题" class="headerlink" title="3.2 webdriver 检测问题"></a>3.2 webdriver 检测问题</h5><p>有些网站还是会检测到是 webdriver 吧，比如淘宝检测到是 webdriver 就会禁止登录了</p>
<p>其实淘宝主要通过 window.navigator.webdriver 来对 webdriver 进行检测，所以我们只需要使用 JavaScript 将它设置为 false 即可，代码如下：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">from</span> pyppeteer <span class="keyword">import</span> launch</span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">def</span> <span class="title">main</span><span class="params">()</span>:</span></span><br><span class="line">    browser = <span class="keyword">await</span> launch(headless=<span class="literal">False</span>, args=[<span class="string">'--disable-infobars'</span>])</span><br><span class="line">    page = <span class="keyword">await</span> browser.newPage()</span><br><span class="line">    <span class="keyword">await</span> page.goto(<span class="string">'https://login.taobao.com/member/login.jhtml?redirectURL=https://www.taobao.com/'</span>)</span><br><span class="line">    <span class="keyword">await</span> page.evaluate(</span><br><span class="line">        <span class="string">'''() =&gt;&#123; Object.defineProperties(navigator,&#123; webdriver:&#123; get: () =&gt; false &#125; &#125;) &#125;'''</span>)</span><br><span class="line">    <span class="keyword">await</span> asyncio.sleep(<span class="number">100</span>)</span><br><span class="line"></span><br><span class="line">asyncio.get_event_loop().run_until_complete(main())</span><br></pre></td></tr></table></figure>

<h5 id="3-3-保持用户记录"><a href="#3-3-保持用户记录" class="headerlink" title="3.3 保持用户记录"></a>3.3 保持用户记录</h5><p>很多朋友在每次启动 Selenium 或 Pyppeteer 的时候总是是一个全新的浏览器，那就是没有设置用户目录，如果设置了它，每次打开就不再是一个全新的浏览器了，它可以恢复之前的历史记录，也可以恢复很多网站的登录信息。那么这个怎么来做呢？很简单，在启动的时候设置 userDataDir 就好了，示例如下：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">from</span> pyppeteer <span class="keyword">import</span> launch</span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">def</span> <span class="title">main</span><span class="params">()</span>:</span></span><br><span class="line">    browser = <span class="keyword">await</span> launch(headless=<span class="literal">False</span>, userDataDir=<span class="string">'./userdata'</span>, args=[<span class="string">'--disable-infobars'</span>])</span><br><span class="line">    page = <span class="keyword">await</span> browser.newPage()</span><br><span class="line">    <span class="keyword">await</span> page.goto(<span class="string">'https://www.taobao.com'</span>)</span><br><span class="line">    <span class="keyword">await</span> asyncio.sleep(<span class="number">100</span>)</span><br><span class="line"></span><br><span class="line">asyncio.get_event_loop().run_until_complete(main())</span><br></pre></td></tr></table></figure>

<h3 id="4-参考资料"><a href="#4-参考资料" class="headerlink" title="4. 参考资料"></a>4. 参考资料</h3><ul>
<li><a href="https://github.com/miyakogi/pyppeteer" target="_blank" rel="noopener">https://github.com/miyakogi/pyppeteer</a></li>
<li><a href="https://juejin.im/post/5c35944b6fb9a049de6d8dd2" target="_blank" rel="noopener">Python爬虫入门教程 24-100 微医挂号网医生数据抓取</a></li>
<li><a href="https://zhuanlan.zhihu.com/p/63634783" target="_blank" rel="noopener">Python中与selenium齐名的pyppeteer库</a></li>
<li><a href="https://www.sanfenzui.com/pyppeteer-bug-collection.html" target="_blank" rel="noopener">pyppeteer使用遇到的bug及解决方法</a></li>
<li><a href="https://juejin.im/post/59e5a86c51882578bf185dba" target="_blank" rel="noopener">https://juejin.im/post/59e5a86c51882578bf185dba</a></li>
</ul>

    </div>

    
    
    
        
      
        <div id="reward-container">
  <div>坚持原创技术分享，您的支持将鼓励我继续创作, 一个耿直的笑脸~</div>
  <button id="reward-button" disable="enable" onclick="var qr = document.getElementById(&quot;qr&quot;); qr.style.display = (qr.style.display === 'none') ? 'block' : 'none';">
    打赏
  </button>
  <div id="qr" style="display: none;">
        
      
      <div style="display: inline-block">
        <img src="/images/wechatpay.jpg" alt="Levon 微信支付">
        <p>微信支付</p>
      </div>
        
      
      <div style="display: inline-block">
        <img src="/images/alipay.jpg" alt="Levon 支付宝">
        <p>支付宝</p>
      </div>

  </div>
</div>

      

      <footer class="post-footer">
          
            
          
          <div class="post-tags">
            
              <a href="/tags/python/" rel="tag"># python</a>
            
              <a href="/tags/pyppeteer/" rel="tag"># pyppeteer</a>
            
              <a href="/tags/爬虫/" rel="tag"># 爬虫</a>
            
          </div>
        

        

          <div class="post-nav">
            <div class="post-nav-next post-nav-item">
              
                <a href="/p/2412099c.html" rel="next" title="python爬虫基础">
                  <i class="fa fa-chevron-left"></i> python爬虫基础
                </a>
              
            </div>

            <span class="post-nav-divider"></span>

            <div class="post-nav-prev post-nav-item">
              
                <a href="/p/9dad86f4.html" rel="prev" title="网络接口介绍">
                  网络接口介绍 <i class="fa fa-chevron-right"></i>
                </a>
              
            </div>
          </div>
        
      </footer>
    
  </div>
  
  
  
  </article>

  </div>


          </div>
          
    
    
  <div class="comments" id="comments">
    <div id="disqus_thread">
      <noscript>Please enable JavaScript to view the comments powered by Disqus.</noscript>
    </div>
  </div>
  
  

        </div>
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">
        
        
        
        
      

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          文章目录
        </li>
        <li class="sidebar-nav-overview">
          站点概览
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
          <div class="post-toc motion-element"><ol class="nav"><li class="nav-item nav-level-3"><a class="nav-link" href="#0-前言"><span class="nav-number">1.</span> <span class="nav-text">0. 前言</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-pyppeteer使用"><span class="nav-number">2.</span> <span class="nav-text">1. pyppeteer使用</span></a><ol class="nav-child"><li class="nav-item nav-level-5"><a class="nav-link" href="#1-1-无头模式"><span class="nav-number">2.0.1.</span> <span class="nav-text">1.1 无头模式</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#1-2-page方法"><span class="nav-number">2.0.2.</span> <span class="nav-text">1.2 page方法</span></a></li></ol></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#2-pyppeteer-linux运行问题"><span class="nav-number">3.</span> <span class="nav-text">2. pyppeteer  linux运行问题</span></a><ol class="nav-child"><li class="nav-item nav-level-5"><a class="nav-link" href="#2-1-centos无法运行pyppeteer"><span class="nav-number">3.0.1.</span> <span class="nav-text">2.1 centos无法运行pyppeteer</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#2-2-Bad-NaCl-helper-startup-ack"><span class="nav-number">3.0.2.</span> <span class="nav-text">2.2 Bad NaCl helper startup ack</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#2-3-Navigation-Timeout-Exceeded-30000-ms-exceeded"><span class="nav-number">3.0.3.</span> <span class="nav-text">2.3 Navigation Timeout Exceeded: 30000 ms exceeded</span></a></li></ol></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-pyppeteer-问题解决方案"><span class="nav-number">4.</span> <span class="nav-text">3. pyppeteer 问题解决方案</span></a><ol class="nav-child"><li class="nav-item nav-level-5"><a class="nav-link" href="#3-1-抓取js-渲染后的数据"><span class="nav-number">4.0.1.</span> <span class="nav-text">3.1 抓取js 渲染后的数据</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#3-2-webdriver-检测问题"><span class="nav-number">4.0.2.</span> <span class="nav-text">3.2 webdriver 检测问题</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#3-3-保持用户记录"><span class="nav-number">4.0.3.</span> <span class="nav-text">3.3 保持用户记录</span></a></li></ol></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#4-参考资料"><span class="nav-number">5.</span> <span class="nav-text">4. 参考资料</span></a></li></ol></div>
        
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <img class="site-author-image" itemprop="image"
      src="/images/avatar.jpg"
      alt="Levon">
  <p class="site-author-name" itemprop="name">Levon</p>
  <div class="site-description" itemprop="description">Never Give Up</div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
        
          <a href="/archives/">
        
          <span class="site-state-item-count">127</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
    
      
      
      <div class="site-state-item site-state-categories">
        
          
            <a href="/categories/">
          
        
        <span class="site-state-item-count">34</span>
        <span class="site-state-item-name">分类</span>
        </a>
      </div>
    
      
      
      <div class="site-state-item site-state-tags">
        
          
            <a href="/tags/">
          
        
        <span class="site-state-item-count">76</span>
        <span class="site-state-item-name">标签</span>
        </a>
      </div>
    
  </nav>
</div>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="https://github.com/unix2dos" title="GitHub &rarr; https://github.com/unix2dos" rel="noopener" target="_blank"><i class="fa fa-fw fa-github"></i>GitHub</a>
      </span>
    
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="mailto:levonfly@gmail.com" title="E-Mail &rarr; mailto:levonfly@gmail.com" rel="noopener" target="_blank"><i class="fa fa-fw fa-envelope"></i>E-Mail</a>
      </span>
    
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="https://twitter.com/levonfly" title="Twitter &rarr; https://twitter.com/levonfly" rel="noopener" target="_blank"><i class="fa fa-fw fa-twitter"></i>Twitter</a>
      </span>
    
      <span class="links-of-author-item">
      
      
        
      
      
        
      
        <a href="https://weibo.com/l6241425" title="Weibo &rarr; https://weibo.com/l6241425" rel="noopener" target="_blank"><i class="fa fa-fw fa-weibo"></i>Weibo</a>
      </span>
    
  </div>



      </div>

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; <span itemprop="copyrightYear">2020</span>
  <span class="with-love" id="animate">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Levon</span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item-icon">
      <i class="fa fa-area-chart"></i>
    </span>
    <span title="站点总字数">507k</span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item-icon">
      <i class="fa fa-coffee"></i>
    </span>
    <span title="站点阅读时长">7:41</span>
</div>

        












        
      </div>
    </footer>
  </div>

  
  <script src="/lib/anime.min.js?v=3.1.0"></script>
  <script src="/lib/velocity/velocity.min.js?v=1.2.1"></script>
  <script src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>
<script src="/js/utils.js?v=7.4.0"></script><script src="/js/motion.js?v=7.4.0"></script>
<script src="/js/schemes/pisces.js?v=7.4.0"></script>

<script src="/js/next-boot.js?v=7.4.0"></script>



  
  <script>
    (function(){
      var bp = document.createElement('script');
      var curProtocol = window.location.protocol.split(':')[0];
      bp.src = (curProtocol === 'https') ? 'https://zz.bdstatic.com/linksubmit/push.js' : 'http://push.zhanzhang.baidu.com/push.js';
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(bp, s);
    })();
  </script>








  <script src="/js/local-search.js?v=7.4.0"></script>














  

  

  

<script>
  function loadCount() {
    var d = document, s = d.createElement('script');
    s.src = 'https://levonfly.disqus.com/count.js';
    s.id = 'dsq-count-scr';
    (d.head || d.body).appendChild(s);
  }
  // defer loading until the whole page loading is completed
  window.addEventListener('load', loadCount, false);
</script>
<script>
  function disqus_config() {
    this.page.url = "https://unix2dos.github.io/p/6b1ba1b4.html";
    this.page.identifier = "p/6b1ba1b4.html";
    this.page.title = 'python爬虫利器pyppeteer的使用';};
  function loadComments() {
    if (window.DISQUS) {
      DISQUS.reset({
        reload: true,
        config: disqus_config
      });
    } else {
      var d = document, s = d.createElement('script');
      s.src = 'https://levonfly.disqus.com/embed.js';
      s.setAttribute('data-timestamp', '' + +new Date());
      (d.head || d.body).appendChild(s);
    }
  }
    window.addEventListener('load', loadComments, false);
  
</script>

</body>
</html>
